import os
from Tools import ParaSplit
import shutil
import Constant

MaxCnt=200

def ParseData(dataPath):

    filelist=os.listdir(dataPath)

    if os.path.exists('Cache/'+dataPath) == False:
        os.makedirs('.//Cache/'+dataPath)
    cachelist=os.listdir('Cache/'+dataPath)
    if len(cachelist) >0 :
        id=int (max(cachelist,key=lambda x:int(x.split('.')[0])).split('.')[0])
    else :
        id=int(0)


    vectors=[]
    for x in filelist:
        name=dataPath+x
        try:            
            file=open(name,'r',encoding='UTF-8')
            paras=ParaSplit.ParaSplit(file.read())
            


            
            for par in paras:
                
                id=id+1
                f = open('.//Cache/'+dataPath+ str(id) + '.txt','w',encoding='UTF-8')
                for p in par:
                    f.write(p+'\n')
                f.close()
                if id >=MaxCnt:
                    break
            file.close()
            if id >=MaxCnt:
                break
        except IOError :
            print ('Error:IO')

        
        

def ClearCache():
    filelist=os.listdir('Cache')
    for x in filelist:
        shutil.rmtree('Cache/'+x)
#ClearCache()
ParseData(Constant.LowLevelSource)
ParseData(Constant.MidLevelSource)
ParseData(Constant.HighLevelSource)
